<?php
require_once __DIR__.'/rssCollector.php';
class XinhuaRssCollector extends rssCollector{

	//构造函数，初始化资源列表和上次抓取时间
    public function __construct() {
		$this->dataPath = storage_path().'/collector/xinhuaData.php';
        $this->rssSourceMap = $this->getCategory();
		$this->categoryMap = $this->getCategoryMap();
        $this->lastGetTime = $this->getLastGetTime();
    }
	   
    //执行抓取
    public function execut() {
        $beginTotalTime = microtime(true);
        //每个大类依次处理
        foreach ($this->rssSourceMap as $category){
			//匹配出频道
			preg_match("/^http:\/\/www\.xinhuanet\.com\/(.*?)\//is", $category['xmlUrl'], $matches);
			$channel = isset($matches[1]) ? trim($matches[1]) : ''; 
            //获取次类列表,不包含详细内容，关键词等
            $listSimple = $this->getList($category['xmlUrl'], $category['title'], $channel);
            //分割数组，匹配关键词、简介、详情
			$listsSimple = array_chunk($listSimple, 30);
			$listDetail = array();
			foreach ($listsSimple as $list) {
				$listDetail = array_merge($listDetail, $this->getDetail($list));
			}
            //保存至数据库
            $beginSaveTime = microtime(true);
            $this->saveToDB($listDetail);
            $endSaveTime = microtime(true);
            $this->saveTime = $endSaveTime - $beginSaveTime;
        }
        $endTotalTime = microtime(true);
        $this->totalTime = $endTotalTime - $beginTotalTime;
        $this->setLastGetTime();
    }
    
    //获取给定分类的简单列表
    private function getList($xmlUrl, $categoryName, $channel = ''){
        $beginTransferTime = microtime(true);
		$listXML = $this->simpleRequest($xmlUrl);
        $endTransferTime = microtime(true);
        $this->transferTime += $endTransferTime - $beginTransferTime;
        //结果过滤+格式整理
        $beginParseTime = microtime(true);
		$parten = '/((?:Mon|Tue|Wed|Thu|Fri|Sat|Sun),\d{1,2}-(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-\d\d\d\d \d\d:\d\d:\d\d) GMT/i';
		$listXML = preg_replace($parten, '<pubDate>$1</pubDate>', $listXML);
		$newsListArrRaw = $this->convertXMLToArray($listXML);
		$newsListArr = isset($newsListArrRaw['channel']['item']) ? $newsListArrRaw['channel']['item'] : array();
		foreach ($newsListArr as $key => $newsSimple){
			//从数组删除数据
			unset($newsListArr[$key]);
			//没有日期的直接放弃
			if(!$newsSimple['pubDate']){
				$this->matchErrorCount += 1;
				continue;
			}
			$newsItem = array();
			$newsPubTime = $newsItem['pubDate'] = strtotime($newsSimple['pubDate']);
			//忽略已经抓过的
			if($newsPubTime < $this->lastGetTime){
				$this->ignoreCount += 1;
				continue;
			}
			//唯一标识
			$newsItem['news_id'] = 'xinhua_'.$newsSimple['link'];
			//标记来源
			$newsItem['source'] = 'xinhua';
			//作者
			$newsItem['author'] = 'xinhua';
			//链接处理
			$newsItem['link'] = $newsLink = $newsSimple['link'];
			//url为空的直接放弃
			if(!$newsLink){
				$this->matchErrorCount += 1;
				continue;
			}
			//匹配出频道
			$newsItem['channel'] = $channel; 
			//分类
			$newsItem['category'] = $this->getLocalCategory($categoryName, 12);
			//标题格式化
			$newsItem['title'] = strip_tags($newsSimple['title']);
			$newsListArr[$key] = $newsItem;
		}
        $endParseTime = microtime(true);
        $this->parseTime += $endParseTime - $beginParseTime;
        return $newsListArr;
    }
    
    //获取详细内容
    private function getDetail($newsListArr){
		//组装抓取节点
		$nodes = array();
		foreach ($newsListArr as $key => $newsArr){
			$nodes[$key] = $newsArr['link'];
		}
		//获取列表内容
		$beginTransferTime = microtime(true);
		$newsListContent = $this->multipleThreadsRequest($nodes);
		$endTransferTime = microtime(true);
		$this->transferTime += $endTransferTime - $beginTransferTime;
		//匹配内容和关键词
		$beginParseTime = microtime(true);
		foreach ($newsListArr as $key => $value){
			$content = $newsListContent[$key];
			$newsListArr[$key]['slides'] = '';
			$newsListArr[$key]['video'] = '';
			$newsListArr[$key]['keywords'] = $this->matchKeywords($content); 
			$newsListArr[$key]['description'] = $this->matchDescription($content);
			//将图片的相对地址改成绝对地址
			$contentDetail = $this->imgRelToAbs($this->matchContnet($content), $nodes[$key]);
			if(!$contentDetail){
				unset($newsListArr[$key]);
				$this->matchErrorCount += 1;
				continue;
			}
			$newsListArr[$key]['content'] = $contentDetail;
			//提取图片
			$imgs = $this->matchImgs($contentDetail);
			$newsListArr[$key]['img'] = isset($imgs[0]) ? $imgs[0] : '';
			$newsListArr[$key]['imgs'] = implode(';', $imgs);
		}
		$endParseTime = microtime(true);
		$this->parseTime += $endParseTime - $beginParseTime;
		return $newsListArr;
    }
    
    //匹配内容
    private function matchContnet($content, $matchPagenate = false){
		//获取内容片断
		$content = $this->matchDiv($content, '/<div(?:.*?)id="content"(?:.*?)>/i');
		//获取分页内容
		if(!$matchPagenate){
			$content = $this->matchPagenateContent($content);
		}
		return $content;
    }
	
	//匹配分页内容（拼接）
	private function matchPagenateContent($content){
		//获取分页div
		$pagenation = $this->matchDiv($content, '/<div width="100%">/i');
		if(!$pagenation){
			return $content;
		}
		//去分页
		$content = str_replace('<div width="100%">'.$pagenation.'</div>', '', $content);
		$content = preg_replace('/<script(?:.*?)>[\s\S]*?<\/script>/i', '', $content);
		//获取center里面的内容
		$parten = '/<center>([\s\S]*?)<\/center>/i';
		preg_match($parten, $pagenation, $matches);
		$center = isset($matches[1]) ? $matches[1] : '';
		if(!$center){
			return $content;
		}
		//获取分页连接
		$parten = '/<a(?:.*?)href="(.*?)"(?:.*?)class="page-Article".*?>\d+<\/a>/i';
		preg_match_all($parten, $center, $matches);
		$links = isset($matches[1]) ? array_unique($matches[1]) : array();
		if(!$links){
			return $content;
		}
		//组装抓取节点
		$nodes = array();
		foreach ($links as $key => $link){
			$nodes[$key] = $link;
		}
		//获取列表内容
		$beginTransferTime = microtime(true);
		$newsListContent = $this->multipleThreadsRequest($nodes);
		$endTransferTime = microtime(true);
		$this->transferTime += $endTransferTime - $beginTransferTime;
		foreach ($newsListContent as $value){
			$content_temp = $this->matchContnet($value, true);
			$pagenation_temp = $this->matchDiv($content_temp, '/<div width="100%">/i');
			//去分页
			$content_temp = str_replace('<div width="100%">'.$pagenation_temp.'</div>', '', $content_temp);
			$content_temp = preg_replace('/<script(?:.*?)>[\s\S]*?<\/script>/i', '', $content_temp);
			$content .= $content_temp;
		}
		return $content;
	}
	
	//匹配关键词
    private function matchKeywords($content){
        $matches = array();
        preg_match('/<meta name="keywords" content="(.*?)"/is', $content, $matches);
        $resHtml = isset($matches[1]) ? trim($matches[1]) : '';
		$keywordsArr = explode(' ', $resHtml);
        return $keywordsArr ? implode(',', $keywordsArr) : '';
    }
	
	//匹配简介
    private function matchDescription($content){
        $matches = array();
        preg_match('/<meta name="description" content="(.*?)"/is', $content, $matches);
        $resHtml = isset($matches[1]) ? trim($matches[1]) : '';
        return strip_tags($resHtml);
    }
	
	//刷新新华网rss源
	public function flushCategoryMap(){
		$category = array(
			0 => array(
				'title' => '时政频道',
				'xmlUrl' => 'http://www.xinhuanet.com/politics/news_politics.xml'
			),
			1 => array(
				'title' => '国际频道',
				'xmlUrl' => 'http://www.xinhuanet.com/world/news_world.xml'
			),
			2 => array(
				'title' => '地方联播',
				'xmlUrl' => 'http://www.xinhuanet.com/local/news_province.xml'
			),
			3 => array(
				'title' => '军事频道',
				'xmlUrl' => 'http://www.xinhuanet.com/mil/news_mil.xml'
			),
			4 => array(
				'title' => '图片频道',
				'xmlUrl' => 'http://www.xinhuanet.com/photo/news_photo.xml'
			),
			/*5 => array(
				'title' => '台湾频道',
				'xmlUrl' => 'http://www.xinhuanet.com/tw/news_tw.xml'
			),有问题，暂时屏蔽*/
			6 => array(
				'title' => '华人频道',
				'xmlUrl' => 'http://www.xinhuanet.com/overseas/news_overseas.xml'
			),
			7 => array(
				'title' => '金融频道',
				'xmlUrl' => 'http://www.xinhuanet.com/finance/news_finance.xml'
			),
			8 => array(
				'title' => '财经频道',
				'xmlUrl' => 'http://www.xinhuanet.com/fortune/news_fortune.xml'
			),
			/*9 => array(
				'title' => '房产频道',
				'xmlUrl' => 'http://www.xinhuanet.com/house/news_house.xml'
			),房产频道长时间停止更新，机构有误，不能抓取*/
			10 => array(
				'title' => '法制频道',
				'xmlUrl' => 'http://www.xinhuanet.com/legal/news_legal.xml'
			),
			11 => array(
				'title' => '教育频道',
				'xmlUrl' => 'http://www.xinhuanet.com/edu/news_edu.xml'
			),
			12 => array(
				'title' => '体育频道',
				'xmlUrl' => 'http://www.xinhuanet.com/sports/news_sports.xml'
			),
			13 => array(
				'title' => '娱乐频道',
				'xmlUrl' => 'http://www.xinhuanet.com/ent/news_ent.xml'
			),
			14 => array(
				'title' => '汽车频道',
				'xmlUrl' => 'http://www.xinhuanet.com/auto/news_auto.xml'
			),
			15 => array(
				'title' => '收藏频道',
				'xmlUrl' => 'http://www.xinhuanet.com/collection/news_collection.xml'
			),
			/*16 => array(
				'title' => '旅游频道',
				'xmlUrl' => 'http://www.xinhuanet.com/travel/news_travel.xml'
			),机构不规则，暂不予抓取*/
			17 => array(
				'title' => '科技频道',
				'xmlUrl' => 'http://www.xinhuanet.com/tech/news_tech.xml'
			),
			18 => array(
				'title' => '出国频道',
				'xmlUrl' => 'http://www.xinhuanet.com/abroad/news_abroad.xml'
			),
			19 => array(
				'title' => '食品频道',
				'xmlUrl' => 'http://www.xinhuanet.com/food/news_food.xml'
			),
			20 => array(
				'title' => '健康频道',
				'xmlUrl' => 'http://www.xinhuanet.com/health/news_health.xml'
			),
			21 => array(
				'title' => '书画频道',
				'xmlUrl' => 'http://www.xinhuanet.com/shuhua/news_calligraphy.xml'
			),
			22 => array(
				'title' => '能源频道',
				'xmlUrl' => 'http://www.xinhuanet.com/energy/news_energy.xml'
			),
			23 => array(
				'title' => '民航频道',
				'xmlUrl' => 'http://www.xinhuanet.com/air/news_air.xml'
			),
		);
		$data = require $this->dataPath;
		$data['category'] = $category;
		$data_str = '<?php return '.var_export($data, true).';';
        return file_put_contents($this->dataPath, $data_str);
	}
}